import json
import os
import sys

import click
curPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.abspath(os.path.join(curPath, '../../../xjlib-common')))    # 插入
print('*** sys.path: %s' % sys.path)
from xjlibrary.database_moudel.simple.mysqlclient import SelctSqlFromDB, MySqlDbConnect
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)


@click.command()
@click.option('--inputdir', default='', type=str, help='请输入需要转换的文件的目录')
@click.option('--outputdir', default='', type=str, help='请输入输出目录')
def file_line_add(inputdir, outputdir):
    print(inputdir)
    print(outputdir)
    for filePath in BaseDir.get_dir_all_files(inputdir):
        print(filePath)
        filename = BaseDir.get_file_name(filePath)
        outfile = BaseDir.get_new_path(outputdir, filename)
        dicts2 = {}
        dicts1 = {}
        listdoi = []
        article_line = ''
        num = 0
        for line in BaseDir.read_file_r_mode_yield(filePath):
            num += 1
            dicts = json.loads(line)
            if not dicts["absHtml"]:
                continue
            doi = dicts["doi"]
            dicts2[doi] = line
            listdoi.append(doi)
            if len(listdoi) > 2000:
                print(num)
                sql = "select `doi`,`jsonmsg` from article where doi in {}".format(tuple(listdoi))
                rows = SelctSqlFromDB(sql, MySqlDbConnect(curPath, "db.ini"))
                if rows:
                    for row in rows:
                        doi = row[0]
                        dicts1[doi] = row[1]
                for key in dicts2.keys():
                    if key not in dicts1.keys():
                        BaseDir.single_add_file("./test.txt", key)
                        continue
                    dicts = json.loads(dicts2[key])
                    dicts["jsonmsg"] = dicts1[key]
                    print(dicts1[key])
                    article_line += json.dumps(dicts)+'\n'
                BaseDir.single_add_file(outfile, article_line)
                dicts2 = {}
                dicts1 = {}
                listdoi = []
                article_line = ''
        print(num)
        sql = "select `doi`,`jsonmsg` from article where doi in {}".format(tuple(listdoi))
        rows = SelctSqlFromDB(sql, MySqlDbConnect(curPath, "db.ini"))
        if rows:
            for row in rows:
                doi = row[0]
                dicts1[doi] = rows[1]
        for key in dicts2.keys():
            if key not in dicts1.keys():
                BaseDir.single_add_file("./test.txt", key)
                continue
            dicts = json.loads(dicts2[key])
            dicts["jsonmsg"] = dicts1[key]
            article_line += json.dumps(dicts) + '\n'
        BaseDir.single_add_file(outfile, article_line)


if __name__ == "__main__":
    file_line_add()
